{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[400]\tvalid_0's f1_score: 0.996235\tvalid_1's f1_score: 0.811245\n",
      "[800]\tvalid_0's f1_score: 0.996235\tvalid_1's f1_score: 0.817052\n",
      "[1200]\tvalid_0's f1_score: 0.996235\tvalid_1's f1_score: 0.816008\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0      0.804     0.872     0.836       859\n",
      "           1      0.838     0.757     0.796       754\n",
      "\n",
      "    accuracy                          0.818      1613\n",
      "   macro avg      0.821     0.815     0.816      1613\n",
      "weighted avg      0.820     0.818     0.817      1613\n",
      "\n",
      "[[749 110]\n",
      " [183 571]]\n"
     ]
    }
   ],
   "source": [
    "#!pip3 install --quiet tensorflow-hub\n",
    "import warnings,time\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import f1_score\n",
    "from lightgbm import LGBMClassifier\n",
    "\n",
    "target = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')['target']\n",
    "train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')[['text']]\n",
    "test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')[['text']]\n",
    "ssub = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')\n",
    "\n",
    "import tensorflow_hub as hub\n",
    "embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/3\")\n",
    "X_train_embeddings = embed(train.text.values)\n",
    "X_test_embeddings = embed(test.text.values)\n",
    "\n",
    "def f1_metric(ytrue,preds):\n",
    "    return 'f1_score', f1_score((preds>=0.5).astype('int'), ytrue, average='macro'), True\n",
    "\n",
    "params = {\n",
    "    'learning_rate': 0.06,\n",
    "    'n_estimators': 1500,\n",
    "    'colsample_bytree': 0.5,\n",
    "    'metric': 'f1_score'\n",
    "}\n",
    "\n",
    "full_clf = LGBMClassifier(**params)\n",
    "\n",
    "full_clf.fit(X_train_embeddings['outputs'][:6000,:], target.values[:6000],\n",
    "             eval_set=[(X_train_embeddings['outputs'][:6000,:], target.values[:6000]),\n",
    "                       (X_train_embeddings['outputs'][6000:,:], target.values[6000:])],\n",
    "             verbose=400, eval_metric=f1_metric\n",
    "            )\n",
    "\n",
    "Y_pred = full_clf.predict(X_train_embeddings['outputs'][6000:])\n",
    "\n",
    "from sklearn import metrics\n",
    "print(metrics.classification_report(target[6000:], Y_pred,  digits=3),)\n",
    "print(metrics.confusion_matrix(target[6000:], Y_pred))\n",
    "\n",
    "full_clf = LGBMClassifier(**params)\n",
    "full_clf.fit(X_train_embeddings['outputs'], target.values)\n",
    "pred_test = full_clf.predict(X_test_embeddings['outputs'])\n",
    "\n",
    "ssub[\"target\"] = pred_test\n",
    "ssub.to_csv(\"submission.csv\",index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-03-07T10:39:13.056Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow_hub as hub\n",
    "# embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/3\")\n",
    "embed = hub.load(\"https://hub.tensorflow.google.cn/google/universal-sentence-encoder/4\")\n",
    "# embed = hub.load(\"./4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1_score 0.814 params {'n_estimators': 1400, 'colsample_bytree': 0.6957, 'learning_rate': 0.065}\n",
      "f1_score 0.816 params {'n_estimators': 1400, 'colsample_bytree': 0.5838, 'learning_rate': 0.021}\n",
      "f1_score 0.802 params {'n_estimators': 600, 'colsample_bytree': 0.6298, 'learning_rate': 0.03}\n",
      "f1_score 0.812 params {'n_estimators': 200, 'colsample_bytree': 0.9658, 'learning_rate': 0.048}\n",
      "f1_score 0.810 params {'n_estimators': 200, 'colsample_bytree': 0.3005, 'learning_rate': 0.03}\n",
      "f1_score 0.821 params {'n_estimators': 800, 'colsample_bytree': 0.4169, 'learning_rate': 0.051}\n",
      "f1_score 0.814 params {'n_estimators': 800, 'colsample_bytree': 0.9755, 'learning_rate': 0.046}\n",
      "f1_score 0.817 params {'n_estimators': 2000, 'colsample_bytree': 0.727, 'learning_rate': 0.03}\n",
      "f1_score 0.809 params {'n_estimators': 600, 'colsample_bytree': 0.5005, 'learning_rate': 0.034}\n",
      "f1_score 0.810 params {'n_estimators': 2000, 'colsample_bytree': 0.7807, 'learning_rate': 0.061}\n",
      "hyopt optimum {'colsample_bytree': 0.6297538462403605, 'learning_rate': 20, 'n_estimators': 600.0}\n",
      "CPU times: user 1h 55min 20s, sys: 53.7 s, total: 1h 56min 14s\n",
      "Wall time: 29min 30s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, ShuffleSplit\n",
    "from sklearn.metrics import make_scorer, f1_score\n",
    "from hyperopt import hp, tpe, fmin\n",
    "from functools import partial\n",
    "from lightgbm import LGBMClassifier\n",
    "\n",
    "def f1_metric(ytrue,preds): \n",
    "    return f1_score((preds>=0.5).astype('int'),ytrue, average='macro')\n",
    "\n",
    "SPACE = {\n",
    "    'n_estimators': hp.quniform('n_estimators', 100, 2000, 200),\n",
    "    'colsample_bytree': hp.uniform('colsample_bytree', 0.3, 1.0),\n",
    "    'learning_rate': hp.choice('learning_rate', np.arange(0.01,0.07,0.001))\n",
    "}\n",
    "\n",
    "def objective(params):\n",
    "    params = {\n",
    "        'n_estimators': int(params['n_estimators']),\n",
    "        'colsample_bytree': round(float(params['colsample_bytree']),4),\n",
    "        'learning_rate': round(float(params['learning_rate']),4),\n",
    "    }\n",
    "\n",
    "    clf = LGBMClassifier(n_jobs=-1,**params)\n",
    "\n",
    "    score = cross_val_score(clf, np.array(X_train_embeddings['outputs']), target.values,\n",
    "                            scoring=make_scorer(f1_metric, greater_is_better=True, needs_proba=False), \n",
    "                            cv=ShuffleSplit(n_splits=4,test_size=.15)).mean()\n",
    "\n",
    "    print(\"f1_score %.3f params %s\"%(score, params))\n",
    "    \n",
    "    return score\n",
    "\n",
    "algo = partial(tpe.suggest, n_startup_jobs=3)\n",
    "\n",
    "best = fmin(fn=objective,space=SPACE,\n",
    "            algo=algo,max_evals=10, \n",
    "            show_progressbar=False, \n",
    "            rstate=np.random.RandomState(64))\n",
    "\n",
    "print(\"hyopt optimum {}\".format(best))\n",
    "\n",
    "best = {'n_estimators': 1400, 'colsample_bytree': 0.475, 'learning_rate': 0.031}\n",
    "opt = {'n_estimators': 400, 'colsample_bytree': 0.710, 'learning_rate': 0.057}\n",
    "\n",
    "\n",
    "# hp searches and returns the best set of hyperparameters without the number of iterations\n",
    "# sources: 1. https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf\n",
    "#          2. https://pdfs.semanticscholar.org/d4f4/9717c9adb46137f49606ebbdf17e3598b5a5.pdf"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
